{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Notebook written by [Zhedong Zheng](https://github.com/zhedongzheng)\n",
    "\n",
    "![alt text](vae.png)\n",
    "\n",
    "Related Papers\n",
    "* [Generating Sentences from a Continuous Space](https://arxiv.org/abs/1511.06349)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from modified import ModifiedBasicDecoder, ModifiedBeamSearchDecoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "PARAMS = {\n",
    "    'max_len': 15,\n",
    "    'embed_dims': 128,\n",
    "    'rnn_size': 128,\n",
    "    'latent_size': 16,\n",
    "    'beam_width': 5,\n",
    "    'num_sampled': 1000,\n",
    "    'clip_norm': 5.0,\n",
    "    'anneal_max': 1.0,\n",
    "    'anneal_bias': 6000,\n",
    "    'batch_size': 128,\n",
    "    'n_epochs': 30,\n",
    "    'word_dropout_rate': 0.8,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_vocab(index_from=4):\n",
    "    PARAMS['word2idx'] = tf.keras.datasets.imdb.get_word_index()\n",
    "    PARAMS['word2idx'] = {k: (v + index_from) for k, v in PARAMS['word2idx'].items()}\n",
    "    PARAMS['word2idx']['<pad>'] = 0\n",
    "    PARAMS['word2idx']['<start>'] = 1\n",
    "    PARAMS['word2idx']['<unk>'] = 2\n",
    "    PARAMS['word2idx']['<end>'] = 3\n",
    "    PARAMS['idx2word'] = {i: w for w, i in PARAMS['word2idx'].items()}\n",
    "    PARAMS['vocab_size'] = len(PARAMS['word2idx'])\n",
    "\n",
    "    \n",
    "def load_data(index_from=4):\n",
    "    (X_train, _), (X_test, _) = tf.contrib.keras.datasets.imdb.load_data(\n",
    "        num_words=None, index_from=index_from)\n",
    "    return (X_train, X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reparam_trick(z_mean, z_logvar):\n",
    "    gaussian = tf.truncated_normal(tf.shape(z_logvar))\n",
    "    z = z_mean + tf.exp(0.5 * z_logvar) * gaussian\n",
    "    return z\n",
    "\n",
    "\n",
    "def kl_w_fn(global_step):\n",
    "    return PARAMS['anneal_max'] * tf.sigmoid((10 / PARAMS['anneal_bias']) * (\n",
    "        tf.to_float(global_step) - tf.constant(PARAMS['anneal_bias'] / 2)))\n",
    "\n",
    "\n",
    "def kl_loss_fn(mean, logvar):\n",
    "    loss = 0.5 * tf.reduce_sum(tf.exp(logvar) + tf.square(mean) - 1 - logvar)\n",
    "    return loss / tf.to_float(tf.shape(mean)[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clip_grads(loss):\n",
    "    variables = tf.trainable_variables()\n",
    "    grads = tf.gradients(loss, variables)\n",
    "    clipped_grads, _ = tf.clip_by_global_norm(grads, PARAMS['clip_norm'])\n",
    "    return zip(clipped_grads, variables)\n",
    "\n",
    "\n",
    "def rnn_cell():\n",
    "    return tf.nn.rnn_cell.GRUCell(PARAMS['rnn_size'],\n",
    "                                  kernel_initializer=tf.orthogonal_initializer())\n",
    "\n",
    "\n",
    "def forward(inputs, labels, mode):\n",
    "    is_training = (mode == tf.estimator.ModeKeys.TRAIN)\n",
    "    enc_seq_len = tf.count_nonzero(inputs, 1, dtype=tf.int32)\n",
    "    batch_sz = tf.shape(inputs)[0]\n",
    "    \n",
    "    with tf.variable_scope('Encoder'):\n",
    "        embedding = tf.get_variable('lookup_table', [len(PARAMS['word2idx']), PARAMS['embed_dims']])\n",
    "        x = tf.nn.embedding_lookup(embedding, inputs)\n",
    "        \n",
    "        _, enc_state = tf.nn.dynamic_rnn(rnn_cell(), x, enc_seq_len, dtype=tf.float32)\n",
    "        \n",
    "        z_mean = tf.layers.dense(enc_state, PARAMS['latent_size'])\n",
    "        z_logvar = tf.layers.dense(enc_state, PARAMS['latent_size'])\n",
    "        \n",
    "    z = reparam_trick(z_mean, z_logvar)\n",
    "        \n",
    "    with tf.variable_scope('Decoder'):\n",
    "        init_state = tf.layers.dense(z, PARAMS['rnn_size'], tf.nn.elu)\n",
    "        output_proj = tf.layers.Dense(len(PARAMS['word2idx']), _scope='decoder/output_proj')\n",
    "        dec_cell = rnn_cell()\n",
    "        \n",
    "        if is_training:\n",
    "            dec_seq_len = tf.count_nonzero(labels['dec_out'], 1, dtype=tf.int32)\n",
    "            \n",
    "            helper = tf.contrib.seq2seq.TrainingHelper(\n",
    "                inputs = tf.nn.embedding_lookup(embedding, labels['dec_inp']),\n",
    "                sequence_length = dec_seq_len)\n",
    "            decoder = ModifiedBasicDecoder(\n",
    "                cell = dec_cell,\n",
    "                helper = helper,\n",
    "                initial_state = init_state,\n",
    "                concat_z = z)\n",
    "            decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                decoder = decoder,\n",
    "                maximum_iterations = tf.reduce_max(dec_seq_len))\n",
    "            rnn_output = decoder_output.rnn_output\n",
    "            \n",
    "            return rnn_output, output_proj(rnn_output), (z_mean, z_logvar)\n",
    "        else:\n",
    "            tiled_z = tf.tile(tf.expand_dims(z, 1), [1, PARAMS['beam_width'], 1])\n",
    "            \n",
    "            decoder = ModifiedBeamSearchDecoder(\n",
    "                cell = dec_cell,\n",
    "                embedding = embedding,\n",
    "                start_tokens = tf.tile(tf.constant([PARAMS['word2idx']['<start>']], tf.int32),\n",
    "                                       [batch_sz]),\n",
    "                end_token = PARAMS['word2idx']['<end>'],\n",
    "                initial_state = tf.contrib.seq2seq.tile_batch(init_state, PARAMS['beam_width']),\n",
    "                beam_width = PARAMS['beam_width'],\n",
    "                output_layer = output_proj,\n",
    "                concat_z = tiled_z)\n",
    "            decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                decoder = decoder)\n",
    "            \n",
    "            return decoder_output.predicted_ids[:, :, 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_fn(features, labels, mode):\n",
    "    logits_or_ids = forward(features, labels, mode)        \n",
    "    \n",
    "    if mode == tf.estimator.ModeKeys.PREDICT:\n",
    "        return tf.estimator.EstimatorSpec(mode, predictions=logits_or_ids)\n",
    "        \n",
    "    if mode == tf.estimator.ModeKeys.TRAIN:\n",
    "        rnn_output, logits, (z_mean, z_logvar) = logits_or_ids\n",
    "        \n",
    "        global_step = tf.train.get_global_step()\n",
    "        \n",
    "        with tf.variable_scope('Decoder/decoder/output_proj', reuse=True):\n",
    "            _weights = tf.transpose(tf.get_variable('kernel'))\n",
    "            _biases = tf.get_variable('bias')\n",
    "        \n",
    "        mask = tf.reshape(tf.to_float(tf.sign(labels['dec_out'])), [-1])\n",
    "        \n",
    "        nll_loss = tf.reduce_sum(mask * tf.nn.sampled_softmax_loss(\n",
    "            weights = _weights,\n",
    "            biases = _biases,\n",
    "            labels = tf.reshape(labels['dec_out'], [-1, 1]),\n",
    "            inputs = tf.reshape(rnn_output, [-1, PARAMS['rnn_size']]),\n",
    "            num_sampled = PARAMS['num_sampled'],\n",
    "            num_classes = PARAMS['vocab_size'],\n",
    "        )) / tf.to_float(tf.shape(features)[0])\n",
    "        \n",
    "        kl_w = kl_w_fn(global_step)\n",
    "        \n",
    "        kl_loss = kl_loss_fn(z_mean, z_logvar)\n",
    "        \n",
    "        loss_op = nll_loss + kl_w * kl_loss\n",
    "        \n",
    "        train_op = tf.train.AdamOptimizer().apply_gradients(\n",
    "            clip_grads(loss_op),\n",
    "            global_step = global_step)\n",
    "        \n",
    "        lth = tf.train.LoggingTensorHook(\n",
    "            {'nll_loss': nll_loss, 'kl_w': kl_w, 'kl_loss': kl_loss}, every_n_iter=100)\n",
    "        \n",
    "        return tf.estimator.EstimatorSpec(\n",
    "            mode=mode, loss=loss_op, train_op=train_op, training_hooks=[lth])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def inf_inp(test_strs):\n",
    "    x = [[PARAMS['word2idx'].get(w, 2) for w in s.split()] for s in test_strs]\n",
    "    x = tf.keras.preprocessing.sequence.pad_sequences(\n",
    "        x, PARAMS['max_len'], truncating='post', padding='post')\n",
    "    return x\n",
    "\n",
    "\n",
    "def demo(test_strs, pred_ids):\n",
    "    for s, pred in zip(test_strs, pred_ids):\n",
    "        print('\\nOriginal:', s)\n",
    "        print('Reconstr:', ' '.join([PARAMS['idx2word'].get(idx, '<unk>') for idx in pred]))\n",
    "\n",
    "        \n",
    "def word_dropout(x):\n",
    "    is_dropped = np.random.binomial(1, PARAMS['word_dropout_rate'], x.shape)\n",
    "    fn = np.vectorize(lambda x, k: PARAMS['word2idx']['<unk>'] if (\n",
    "                      k and (x not in range(4))) else x)\n",
    "    return fn(x, is_dropped)\n",
    "        \n",
    "        \n",
    "def main():\n",
    "    build_vocab()\n",
    "    \n",
    "    X = np.concatenate(load_data())\n",
    "    X = np.concatenate((tf.keras.preprocessing.sequence.pad_sequences(\n",
    "                            X, PARAMS['max_len'], truncating='post', padding='post'),\n",
    "                        tf.keras.preprocessing.sequence.pad_sequences(\n",
    "                            X, PARAMS['max_len'], truncating='pre', padding='post')))\n",
    "\n",
    "    enc_inp = X[:, 1:]\n",
    "    dec_inp = X[:]\n",
    "    dec_out = np.concatenate([X[:, 1:], np.full([X.shape[0], 1], PARAMS['word2idx']['<end>'])], 1)\n",
    "\n",
    "    test_strs = ['i love this film and i think it is one of the best films',\n",
    "                 'this movie is a waste of time and there is no point to watch it']\n",
    "\n",
    "    estimator = tf.estimator.Estimator(model_fn)\n",
    "\n",
    "    for _ in range(PARAMS['n_epochs']):\n",
    "        estimator.train(tf.estimator.inputs.numpy_input_fn(\n",
    "            x = enc_inp,\n",
    "            y = {'dec_inp': word_dropout(dec_inp), 'dec_out': dec_out},\n",
    "            batch_size = PARAMS['batch_size'],\n",
    "            shuffle = True))\n",
    "\n",
    "        pred_ids = list(estimator.predict(tf.estimator.inputs.numpy_input_fn(\n",
    "            x = inf_inp(test_strs),\n",
    "            shuffle = False)))\n",
    "\n",
    "        demo(test_strs, pred_ids)\n",
    "        print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using default config.\n",
      "WARNING:tensorflow:Using temporary folder as model directory: /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a\n",
      "INFO:tensorflow:Using config: {'_model_dir': '/var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': None, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x121dd76d8>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/site-packages/tensorflow/python/ops/nn_impl.py:1344: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See @{tf.nn.softmax_cross_entropy_with_logits_v2}.\n",
      "\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 1 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:loss = 155.36774, step = 1\n",
      "INFO:tensorflow:nll_loss = 155.36774, kl_w = 0.006692851, kl_loss = 4.205638e-05\n",
      "INFO:tensorflow:global_step/sec: 2.81709\n",
      "INFO:tensorflow:loss = 87.78328, step = 101 (35.499 sec)\n",
      "INFO:tensorflow:nll_loss = 86.716995, kl_w = 0.007897083, kl_loss = 135.02206 (35.499 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.84305\n",
      "INFO:tensorflow:loss = 86.539825, step = 201 (35.173 sec)\n",
      "INFO:tensorflow:nll_loss = 86.06151, kl_w = 0.009315956, kl_loss = 51.344177 (35.174 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.90217\n",
      "INFO:tensorflow:loss = 87.02032, step = 301 (34.457 sec)\n",
      "INFO:tensorflow:nll_loss = 86.57315, kl_w = 0.010986943, kl_loss = 40.700134 (34.457 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.86221\n",
      "INFO:tensorflow:loss = 85.542, step = 401 (34.939 sec)\n",
      "INFO:tensorflow:nll_loss = 85.072365, kl_w = 0.012953726, kl_loss = 36.254654 (34.939 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.88611\n",
      "INFO:tensorflow:loss = 83.35156, step = 501 (34.648 sec)\n",
      "INFO:tensorflow:nll_loss = 82.89714, kl_w = 0.015267149, kl_loss = 29.764515 (34.648 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.79711\n",
      "INFO:tensorflow:loss = 84.218315, step = 601 (35.752 sec)\n",
      "INFO:tensorflow:nll_loss = 83.772255, kl_w = 0.01798621, kl_loss = 24.800219 (35.752 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.84094\n",
      "INFO:tensorflow:loss = 82.26413, step = 701 (35.200 sec)\n",
      "INFO:tensorflow:nll_loss = 81.85507, kl_w = 0.021179108, kl_loss = 19.314379 (35.200 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 782 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 80.72725.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-782\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i is this this this this this i i i i i i the <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: i is this this this this this this i i i i i the <end>\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-782\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 783 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:loss = 79.036934, step = 783\n",
      "INFO:tensorflow:nll_loss = 78.62474, kl_w = 0.024205629, kl_loss = 17.0289\n",
      "INFO:tensorflow:global_step/sec: 2.75229\n",
      "INFO:tensorflow:loss = 81.97445, step = 883 (36.335 sec)\n",
      "INFO:tensorflow:nll_loss = 81.60156, kl_w = 0.028470589, kl_loss = 13.097271 (36.335 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.81881\n",
      "INFO:tensorflow:loss = 80.94466, step = 983 (35.476 sec)\n",
      "INFO:tensorflow:nll_loss = 80.53073, kl_w = 0.033461247, kl_loss = 12.370265 (35.475 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.97962\n",
      "INFO:tensorflow:loss = 78.66729, step = 1083 (33.563 sec)\n",
      "INFO:tensorflow:nll_loss = 78.036415, kl_w = 0.039291352, kl_loss = 16.056316 (33.563 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.04416\n",
      "INFO:tensorflow:loss = 78.4203, step = 1183 (32.849 sec)\n",
      "INFO:tensorflow:nll_loss = 77.74448, kl_w = 0.04608883, kl_loss = 14.663654 (32.849 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.89832\n",
      "INFO:tensorflow:loss = 75.87271, step = 1283 (34.504 sec)\n",
      "INFO:tensorflow:nll_loss = 75.019844, kl_w = 0.053996176, kl_loss = 15.795002 (34.504 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.69885\n",
      "INFO:tensorflow:loss = 72.92844, step = 1383 (37.051 sec)\n",
      "INFO:tensorflow:nll_loss = 71.82825, kl_w = 0.06317033, kl_loss = 17.416245 (37.051 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.81603\n",
      "INFO:tensorflow:loss = 70.05235, step = 1483 (35.511 sec)\n",
      "INFO:tensorflow:nll_loss = 68.59155, kl_w = 0.07378165, kl_loss = 19.798994 (35.511 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 1564 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 70.3634.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-1564\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i saw this movie of i seen this movie it was the time of <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this is a one of the worst movie i seen it and it <end> <end>\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-1564\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 1565 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:loss = 79.092995, step = 1565\n",
      "INFO:tensorflow:nll_loss = 77.579185, kl_w = 0.08368247, kl_loss = 18.089901\n",
      "INFO:tensorflow:global_step/sec: 2.93195\n",
      "INFO:tensorflow:loss = 68.16621, step = 1665 (34.108 sec)\n",
      "INFO:tensorflow:nll_loss = 66.246346, kl_w = 0.09738124, kl_loss = 19.71486 (34.108 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.81883\n",
      "INFO:tensorflow:loss = 66.81917, step = 1765 (35.476 sec)\n",
      "INFO:tensorflow:nll_loss = 64.61618, kl_w = 0.11304584, kl_loss = 19.487585 (35.476 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.82832\n",
      "INFO:tensorflow:loss = 67.46002, step = 1865 (35.357 sec)\n",
      "INFO:tensorflow:nll_loss = 64.86088, kl_w = 0.13086486, kl_loss = 19.86128 (35.357 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.80797\n",
      "INFO:tensorflow:loss = 67.85671, step = 1965 (35.612 sec)\n",
      "INFO:tensorflow:nll_loss = 65.121826, kl_w = 0.15101443, kl_loss = 18.110104 (35.612 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.02205\n",
      "INFO:tensorflow:loss = 70.723495, step = 2065 (33.090 sec)\n",
      "INFO:tensorflow:nll_loss = 67.815414, kl_w = 0.17364664, kl_loss = 16.747131 (33.090 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.87994\n",
      "INFO:tensorflow:loss = 72.51284, step = 2165 (34.723 sec)\n",
      "INFO:tensorflow:nll_loss = 69.306076, kl_w = 0.19887616, kl_loss = 16.124443 (34.723 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 2.89008\n",
      "INFO:tensorflow:loss = 70.74055, step = 2265 (34.602 sec)\n",
      "INFO:tensorflow:nll_loss = 67.16752, kl_w = 0.22676536, kl_loss = 15.756486 (34.602 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 2346 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 67.699715.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-2346\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: it was this movie i have seen it is a lot of the time <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie is the worst movie and ever seen this movie ever seen it <end>\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-2346\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 2347 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:loss = 67.15013, step = 2347\n",
      "INFO:tensorflow:nll_loss = 63.192886, kl_w = 0.25161827, kl_loss = 15.727173\n",
      "INFO:tensorflow:global_step/sec: 2.87378\n",
      "INFO:tensorflow:loss = 67.19423, step = 2447 (34.799 sec)\n",
      "INFO:tensorflow:nll_loss = 62.910873, kl_w = 0.2842792, kl_loss = 15.067433 (34.799 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.90219\n",
      "INFO:tensorflow:loss = 67.72771, step = 2547 (34.457 sec)\n",
      "INFO:tensorflow:nll_loss = 62.93488, kl_w = 0.31937042, kl_loss = 15.007122 (34.457 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.10404\n",
      "INFO:tensorflow:loss = 65.94809, step = 2647 (32.216 sec)\n",
      "INFO:tensorflow:nll_loss = 60.733833, kl_w = 0.35663486, kl_loss = 14.620718 (32.216 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.82075\n",
      "INFO:tensorflow:loss = 66.144684, step = 2747 (35.452 sec)\n",
      "INFO:tensorflow:nll_loss = 60.470276, kl_w = 0.39571938, kl_loss = 14.339466 (35.452 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.91991\n",
      "INFO:tensorflow:loss = 70.74233, step = 2847 (34.248 sec)\n",
      "INFO:tensorflow:nll_loss = 65.15898, kl_w = 0.43618327, kl_loss = 12.800478 (34.247 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.84697\n",
      "INFO:tensorflow:loss = 70.98012, step = 2947 (35.125 sec)\n",
      "INFO:tensorflow:nll_loss = 64.97897, kl_w = 0.47751516, kl_loss = 12.567446 (35.125 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.02538\n",
      "INFO:tensorflow:loss = 69.74873, step = 3047 (33.054 sec)\n",
      "INFO:tensorflow:nll_loss = 63.336487, kl_w = 0.5191573, kl_loss = 12.351266 (33.054 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 3128 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 68.30807.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-3128\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i give this movie and i have it is a lot of the time <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie is the worst movie and ever seen this movie ever seen it <end>\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-3128\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 3129 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:loss = 71.10296, step = 3129\n",
      "INFO:tensorflow:nll_loss = 64.67767, kl_w = 0.55313194, kl_loss = 11.616192\n",
      "INFO:tensorflow:global_step/sec: 2.8178\n",
      "INFO:tensorflow:loss = 67.64965, step = 3229 (35.490 sec)\n",
      "INFO:tensorflow:nll_loss = 60.760452, kl_w = 0.5938731, kl_loss = 11.600459 (35.490 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.94555\n",
      "INFO:tensorflow:loss = 67.36408, step = 3329 (33.950 sec)\n",
      "INFO:tensorflow:nll_loss = 60.092796, kl_w = 0.6333619, kl_loss = 11.480461 (33.949 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.86872\n",
      "INFO:tensorflow:loss = 66.999, step = 3429 (34.860 sec)\n",
      "INFO:tensorflow:nll_loss = 59.48549, kl_w = 0.6711373, kl_loss = 11.195191 (34.860 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.83522\n",
      "INFO:tensorflow:loss = 67.3617, step = 3529 (35.269 sec)\n",
      "INFO:tensorflow:nll_loss = 59.564445, kl_w = 0.7068222, kl_loss = 11.031429 (35.269 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.03665\n",
      "INFO:tensorflow:loss = 72.47289, step = 3629 (32.931 sec)\n",
      "INFO:tensorflow:nll_loss = 65.08754, kl_w = 0.7401343, kl_loss = 9.978395 (32.931 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.78999\n",
      "INFO:tensorflow:loss = 73.52361, step = 3729 (35.843 sec)\n",
      "INFO:tensorflow:nll_loss = 65.91862, kl_w = 0.7708882, kl_loss = 9.865238 (35.843 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.76726\n",
      "INFO:tensorflow:loss = 72.35768, step = 3829 (36.138 sec)\n",
      "INFO:tensorflow:nll_loss = 64.63472, kl_w = 0.79899096, kl_loss = 9.665894 (36.137 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 3910 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 70.16428.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-3910\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i think this movie i think it was a lot of the other time <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie is the worst movie and i have seen it i give it <end>\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-3910\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 3911 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:loss = 70.67818, step = 3911\n",
      "INFO:tensorflow:nll_loss = 63.054028, kl_w = 0.82004714, kl_loss = 9.297213\n",
      "INFO:tensorflow:global_step/sec: 3.00799\n",
      "INFO:tensorflow:loss = 71.754486, step = 4011 (33.246 sec)\n",
      "INFO:tensorflow:nll_loss = 63.709255, kl_w = 0.84334546, kl_loss = 9.539661 (33.246 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.98436\n",
      "INFO:tensorflow:loss = 71.47984, step = 4111 (33.508 sec)\n",
      "INFO:tensorflow:nll_loss = 63.615395, kl_w = 0.8641271, kl_loss = 9.101034 (33.508 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.83514\n",
      "INFO:tensorflow:loss = 70.85768, step = 4211 (35.271 sec)\n",
      "INFO:tensorflow:nll_loss = 62.827736, kl_w = 0.88253593, kl_loss = 9.0987215 (35.271 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.86829\n",
      "INFO:tensorflow:loss = 70.25078, step = 4311 (34.864 sec)\n",
      "INFO:tensorflow:nll_loss = 61.824852, kl_w = 0.89874285, kl_loss = 9.375235 (34.864 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.85604\n",
      "INFO:tensorflow:loss = 70.5507, step = 4411 (35.013 sec)\n",
      "INFO:tensorflow:nll_loss = 61.934223, kl_w = 0.9129343, kl_loss = 9.438223 (35.014 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.00611\n",
      "INFO:tensorflow:loss = 68.30308, step = 4511 (33.265 sec)\n",
      "INFO:tensorflow:nll_loss = 59.898697, kl_w = 0.92530197, kl_loss = 9.0828495 (33.265 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.78475\n",
      "INFO:tensorflow:loss = 68.008705, step = 4611 (35.910 sec)\n",
      "INFO:tensorflow:nll_loss = 59.53071, kl_w = 0.93603605, kl_loss = 9.057337 (35.910 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Saving checkpoints for 4692 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 73.8559.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-4692\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i watched this movie and i have seen it is one of the time <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie was one of the time it was a chance to watch it <end>\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-4692\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 4693 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:loss = 70.54564, step = 4693\n",
      "INFO:tensorflow:nll_loss = 62.08492, kl_w = 0.94374704, kl_loss = 8.96503\n",
      "INFO:tensorflow:global_step/sec: 2.86146\n",
      "INFO:tensorflow:loss = 67.23025, step = 4793 (34.948 sec)\n",
      "INFO:tensorflow:nll_loss = 58.673447, kl_w = 0.95196813, kl_loss = 8.988537 (34.949 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.02405\n",
      "INFO:tensorflow:loss = 67.74285, step = 4893 (33.068 sec)\n",
      "INFO:tensorflow:nll_loss = 58.99769, kl_w = 0.9590399, kl_loss = 9.118666 (33.068 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.99252\n",
      "INFO:tensorflow:loss = 70.67621, step = 4993 (33.417 sec)\n",
      "INFO:tensorflow:nll_loss = 62.364136, kl_w = 0.9651086, kl_loss = 8.61258 (33.417 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.8712\n",
      "INFO:tensorflow:loss = 72.0085, step = 5093 (34.829 sec)\n",
      "INFO:tensorflow:nll_loss = 63.558422, kl_w = 0.97030604, kl_loss = 8.708669 (34.829 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.74208\n",
      "INFO:tensorflow:loss = 71.709435, step = 5193 (36.469 sec)\n",
      "INFO:tensorflow:nll_loss = 63.253452, kl_w = 0.97474945, kl_loss = 8.675034 (36.470 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.71989\n",
      "INFO:tensorflow:loss = 73.44318, step = 5293 (36.766 sec)\n",
      "INFO:tensorflow:nll_loss = 64.77922, kl_w = 0.9785427, kl_loss = 8.853935 (36.766 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.64751\n",
      "INFO:tensorflow:loss = 68.3464, step = 5393 (37.771 sec)\n",
      "INFO:tensorflow:nll_loss = 59.617065, kl_w = 0.9817768, kl_loss = 8.891363 (37.772 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 5474 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 69.49936.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-5474\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i was a movie that i thought it is a lot of the best <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie was one of the time i was my time i have seen <end>\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-5474\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 5475 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:loss = 75.54589, step = 5475\n",
      "INFO:tensorflow:nll_loss = 66.674576, kl_w = 0.98406756, kl_loss = 9.014946\n",
      "INFO:tensorflow:global_step/sec: 2.79512\n",
      "INFO:tensorflow:loss = 69.62484, step = 5575 (35.778 sec)\n",
      "INFO:tensorflow:nll_loss = 60.967487, kl_w = 0.9864804, kl_loss = 8.775999 (35.778 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.78638\n",
      "INFO:tensorflow:loss = 71.61641, step = 5675 (35.889 sec)\n",
      "INFO:tensorflow:nll_loss = 62.9644, kl_w = 0.98853207, kl_loss = 8.7523775 (35.890 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.83988\n",
      "INFO:tensorflow:loss = 67.98294, step = 5775 (35.212 sec)\n",
      "INFO:tensorflow:nll_loss = 58.99628, kl_w = 0.9902755, kl_loss = 9.074907 (35.212 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.03293\n",
      "INFO:tensorflow:loss = 68.74669, step = 5875 (32.972 sec)\n",
      "INFO:tensorflow:nll_loss = 59.788635, kl_w = 0.9917561, kl_loss = 9.032519 (32.972 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.81299\n",
      "INFO:tensorflow:loss = 67.76902, step = 5975 (35.549 sec)\n",
      "INFO:tensorflow:nll_loss = 58.679314, kl_w = 0.99301285, kl_loss = 9.153662 (35.550 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.77201\n",
      "INFO:tensorflow:loss = 67.06563, step = 6075 (36.075 sec)\n",
      "INFO:tensorflow:nll_loss = 57.832764, kl_w = 0.9940791, kl_loss = 9.287857 (36.075 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.91729\n",
      "INFO:tensorflow:loss = 72.08056, step = 6175 (34.278 sec)\n",
      "INFO:tensorflow:nll_loss = 63.614456, kl_w = 0.99498355, kl_loss = 8.508783 (34.278 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 6256 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 72.70856.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-6256\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i love this movie if you can say this one of the best movies <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie was the best movie and it was worth me to watch it <end>\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-6256\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 6257 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:loss = 69.80703, step = 6257\n",
      "INFO:tensorflow:nll_loss = 60.43278, kl_w = 0.9956215, kl_loss = 9.415475\n",
      "INFO:tensorflow:global_step/sec: 2.85395\n",
      "INFO:tensorflow:loss = 69.69832, step = 6357 (35.040 sec)\n",
      "INFO:tensorflow:nll_loss = 60.730614, kl_w = 0.9962913, kl_loss = 9.001084 (35.040 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.92913\n",
      "INFO:tensorflow:loss = 71.34522, step = 6457 (34.141 sec)\n",
      "INFO:tensorflow:nll_loss = 62.418034, kl_w = 0.99685884, kl_loss = 8.955319 (34.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.86959\n",
      "INFO:tensorflow:loss = 71.52009, step = 6557 (34.848 sec)\n",
      "INFO:tensorflow:nll_loss = 62.813995, kl_w = 0.9973398, kl_loss = 8.729314 (34.848 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.98808\n",
      "INFO:tensorflow:loss = 70.436554, step = 6657 (33.465 sec)\n",
      "INFO:tensorflow:nll_loss = 61.595406, kl_w = 0.99774724, kl_loss = 8.861112 (33.465 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.99602\n",
      "INFO:tensorflow:loss = 69.22643, step = 6757 (33.378 sec)\n",
      "INFO:tensorflow:nll_loss = 60.12555, kl_w = 0.99809235, kl_loss = 9.118279 (33.378 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.86884\n",
      "INFO:tensorflow:loss = 67.18539, step = 6857 (34.859 sec)\n",
      "INFO:tensorflow:nll_loss = 58.08656, kl_w = 0.99838483, kl_loss = 9.113545 (34.859 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.76667\n",
      "INFO:tensorflow:loss = 68.29861, step = 6957 (36.143 sec)\n",
      "INFO:tensorflow:nll_loss = 58.965446, kl_w = 0.9986324, kl_loss = 9.345944 (36.143 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 7038 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 67.30334.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-7038\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i love this movie when i have seen it is a lot of the <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this is one of the worst movie i've ever seen it was some and <end>\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-7038\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 7039 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:loss = 66.37, step = 7039\n",
      "INFO:tensorflow:nll_loss = 57.016388, kl_w = 0.99880695, kl_loss = 9.364788\n",
      "INFO:tensorflow:global_step/sec: 2.87359\n",
      "INFO:tensorflow:loss = 67.67604, step = 7139 (34.801 sec)\n",
      "INFO:tensorflow:nll_loss = 58.160973, kl_w = 0.9989899, kl_loss = 9.524687 (34.801 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.93371\n",
      "INFO:tensorflow:loss = 71.354965, step = 7239 (34.086 sec)\n",
      "INFO:tensorflow:nll_loss = 62.01683, kl_w = 0.9991448, kl_loss = 9.34613 (34.086 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.98295\n",
      "INFO:tensorflow:loss = 70.43383, step = 7339 (33.524 sec)\n",
      "INFO:tensorflow:nll_loss = 61.39624, kl_w = 0.999276, kl_loss = 9.044138 (33.524 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.97498\n",
      "INFO:tensorflow:loss = 70.0444, step = 7439 (33.614 sec)\n",
      "INFO:tensorflow:nll_loss = 60.962997, kl_w = 0.999387, kl_loss = 9.086977 (33.614 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.86132\n",
      "INFO:tensorflow:loss = 70.177925, step = 7539 (34.949 sec)\n",
      "INFO:tensorflow:nll_loss = 61.024082, kl_w = 0.99948114, kl_loss = 9.158598 (34.949 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.76652\n",
      "INFO:tensorflow:loss = 69.415665, step = 7639 (36.147 sec)\n",
      "INFO:tensorflow:nll_loss = 60.102203, kl_w = 0.9995608, kl_loss = 9.317556 (36.147 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.7231\n",
      "INFO:tensorflow:loss = 66.65989, step = 7739 (36.723 sec)\n",
      "INFO:tensorflow:nll_loss = 57.05069, kl_w = 0.9996282, kl_loss = 9.612774 (36.723 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 7820 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 67.459885.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-7820\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i loved this movie that i have seen it is one of the worst <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie is a lot of time i've seen that it had a lot <end>\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-7820\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 7821 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:loss = 71.50121, step = 7821\n",
      "INFO:tensorflow:nll_loss = 62.015457, kl_w = 0.99967563, kl_loss = 9.488832\n",
      "INFO:tensorflow:global_step/sec: 2.96613\n",
      "INFO:tensorflow:loss = 66.20829, step = 7921 (33.715 sec)\n",
      "INFO:tensorflow:nll_loss = 56.156525, kl_w = 0.9997254, kl_loss = 10.054529 (33.715 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.04561\n",
      "INFO:tensorflow:loss = 67.09968, step = 8021 (32.835 sec)\n",
      "INFO:tensorflow:nll_loss = 57.392235, kl_w = 0.9997676, kl_loss = 9.7097 (32.835 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.91016\n",
      "INFO:tensorflow:loss = 65.6919, step = 8121 (34.363 sec)\n",
      "INFO:tensorflow:nll_loss = 55.731777, kl_w = 0.99980325, kl_loss = 9.962087 (34.363 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.98279\n",
      "INFO:tensorflow:loss = 65.61524, step = 8221 (33.525 sec)\n",
      "INFO:tensorflow:nll_loss = 56.00663, kl_w = 0.99983346, kl_loss = 9.61021 (33.525 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.79651\n",
      "INFO:tensorflow:loss = 70.11808, step = 8321 (35.759 sec)\n",
      "INFO:tensorflow:nll_loss = 60.883724, kl_w = 0.999859, kl_loss = 9.23566 (35.759 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.01893\n",
      "INFO:tensorflow:loss = 70.23424, step = 8421 (33.124 sec)\n",
      "INFO:tensorflow:nll_loss = 60.814762, kl_w = 0.9998807, kl_loss = 9.420601 (33.124 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.99127\n",
      "INFO:tensorflow:loss = 71.739105, step = 8521 (33.431 sec)\n",
      "INFO:tensorflow:nll_loss = 62.55261, kl_w = 0.9998989, kl_loss = 9.1874275 (33.431 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 8602 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 71.90732.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-8602\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i am the first time i think it was a lot of the book <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie is the worst movie that i've seen this one to be better <end>\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-8602\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 8603 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:loss = 68.128845, step = 8603\n",
      "INFO:tensorflow:nll_loss = 58.15779, kl_w = 0.9999119, kl_loss = 9.97193\n",
      "INFO:tensorflow:global_step/sec: 3.17522\n",
      "INFO:tensorflow:loss = 69.34385, step = 8703 (31.495 sec)\n",
      "INFO:tensorflow:nll_loss = 60.009624, kl_w = 0.9999254, kl_loss = 9.334921 (31.495 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.09034\n",
      "INFO:tensorflow:loss = 69.89823, step = 8803 (32.358 sec)\n",
      "INFO:tensorflow:nll_loss = 60.517704, kl_w = 0.9999368, kl_loss = 9.38112 (32.358 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.10151\n",
      "INFO:tensorflow:loss = 68.68089, step = 8903 (32.242 sec)\n",
      "INFO:tensorflow:nll_loss = 59.32138, kl_w = 0.9999466, kl_loss = 9.36001 (32.243 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.09558\n",
      "INFO:tensorflow:loss = 69.180756, step = 9003 (32.305 sec)\n",
      "INFO:tensorflow:nll_loss = 59.55577, kl_w = 0.9999547, kl_loss = 9.625421 (32.305 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.04276\n",
      "INFO:tensorflow:loss = 67.37028, step = 9103 (32.864 sec)\n",
      "INFO:tensorflow:nll_loss = 57.58039, kl_w = 0.99996173, kl_loss = 9.790263 (32.864 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.05194\n",
      "INFO:tensorflow:loss = 65.42141, step = 9203 (32.766 sec)\n",
      "INFO:tensorflow:nll_loss = 55.64464, kl_w = 0.9999676, kl_loss = 9.777082 (32.766 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.0018\n",
      "INFO:tensorflow:loss = 65.83144, step = 9303 (33.313 sec)\n",
      "INFO:tensorflow:nll_loss = 55.721306, kl_w = 0.9999726, kl_loss = 10.110413 (33.313 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 9384 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 66.15379.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-9384\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i saw this movie and i think it is a lot of the best <end> <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie is a great movie of it is a good and it was <end> <end>\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-9384\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 9385 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:loss = 70.768326, step = 9385\n",
      "INFO:tensorflow:nll_loss = 60.89624, kl_w = 0.99997604, kl_loss = 9.872324\n",
      "INFO:tensorflow:global_step/sec: 2.83266\n",
      "INFO:tensorflow:loss = 64.60375, step = 9485 (35.304 sec)\n",
      "INFO:tensorflow:nll_loss = 54.522102, kl_w = 0.99997973, kl_loss = 10.081855 (35.305 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.96195\n",
      "INFO:tensorflow:loss = 64.51243, step = 9585 (33.761 sec)\n",
      "INFO:tensorflow:nll_loss = 54.34383, kl_w = 0.99998283, kl_loss = 10.168774 (33.760 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.06058\n",
      "INFO:tensorflow:loss = 64.995224, step = 9685 (32.675 sec)\n",
      "INFO:tensorflow:nll_loss = 54.70936, kl_w = 0.99998546, kl_loss = 10.286015 (32.675 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.13336\n",
      "INFO:tensorflow:loss = 65.80724, step = 9785 (31.915 sec)\n",
      "INFO:tensorflow:nll_loss = 55.638702, kl_w = 0.9999877, kl_loss = 10.168668 (31.915 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.96806\n",
      "INFO:tensorflow:loss = 70.83133, step = 9885 (33.691 sec)\n",
      "INFO:tensorflow:nll_loss = 61.29073, kl_w = 0.9999896, kl_loss = 9.540697 (33.691 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.94945\n",
      "INFO:tensorflow:loss = 69.06653, step = 9985 (33.905 sec)\n",
      "INFO:tensorflow:nll_loss = 59.422283, kl_w = 0.9999912, kl_loss = 9.644331 (33.905 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.89125\n",
      "INFO:tensorflow:loss = 67.10641, step = 10085 (34.587 sec)\n",
      "INFO:tensorflow:nll_loss = 57.082497, kl_w = 0.9999925, kl_loss = 10.023985 (34.587 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 10166 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 68.73115.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-10166\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i liked this movie and i was in this one of the best movies <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: the movie is a waste of time i saw it to me to see <end>\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-10166\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 10167 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:loss = 66.01576, step = 10167\n",
      "INFO:tensorflow:nll_loss = 55.64933, kl_w = 0.99999344, kl_loss = 10.366504\n",
      "INFO:tensorflow:global_step/sec: 3.16881\n",
      "INFO:tensorflow:loss = 63.223984, step = 10267 (31.559 sec)\n",
      "INFO:tensorflow:nll_loss = 52.95747, kl_w = 0.9999945, kl_loss = 10.266568 (31.559 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.92502\n",
      "INFO:tensorflow:loss = 63.267437, step = 10367 (34.188 sec)\n",
      "INFO:tensorflow:nll_loss = 52.78791, kl_w = 0.99999535, kl_loss = 10.479574 (34.188 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.17831\n",
      "INFO:tensorflow:loss = 65.434235, step = 10467 (31.463 sec)\n",
      "INFO:tensorflow:nll_loss = 54.986217, kl_w = 0.99999607, kl_loss = 10.448055 (31.464 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.20426\n",
      "INFO:tensorflow:loss = 68.638885, step = 10567 (31.210 sec)\n",
      "INFO:tensorflow:nll_loss = 58.74733, kl_w = 0.99999666, kl_loss = 9.891586 (31.210 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.00042\n",
      "INFO:tensorflow:loss = 67.17908, step = 10667 (33.327 sec)\n",
      "INFO:tensorflow:nll_loss = 57.28483, kl_w = 0.99999714, kl_loss = 9.894276 (33.327 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.0356\n",
      "INFO:tensorflow:loss = 68.63626, step = 10767 (32.942 sec)\n",
      "INFO:tensorflow:nll_loss = 58.79048, kl_w = 0.9999976, kl_loss = 9.845802 (32.942 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.04021\n",
      "INFO:tensorflow:loss = 68.43944, step = 10867 (32.893 sec)\n",
      "INFO:tensorflow:nll_loss = 58.421436, kl_w = 0.999998, kl_loss = 10.018021 (32.893 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 10948 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 65.83031.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-10948\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i saw this movie and i think it was one of the worst movie <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie is the worst movie ever made in this movie has been made <end>\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-10948\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 10949 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:loss = 65.37626, step = 10949\n",
      "INFO:tensorflow:nll_loss = 55.091934, kl_w = 0.9999982, kl_loss = 10.284346\n",
      "INFO:tensorflow:global_step/sec: 3.02799\n",
      "INFO:tensorflow:loss = 66.27994, step = 11049 (33.026 sec)\n",
      "INFO:tensorflow:nll_loss = 55.896248, kl_w = 0.99999845, kl_loss = 10.383705 (33.026 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.19173\n",
      "INFO:tensorflow:loss = 64.925255, step = 11149 (31.331 sec)\n",
      "INFO:tensorflow:nll_loss = 54.541748, kl_w = 0.9999987, kl_loss = 10.383518 (31.331 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.02315\n",
      "INFO:tensorflow:loss = 67.85429, step = 11249 (33.078 sec)\n",
      "INFO:tensorflow:nll_loss = 57.69948, kl_w = 0.9999989, kl_loss = 10.154818 (33.078 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.17562\n",
      "INFO:tensorflow:loss = 66.62067, step = 11349 (31.490 sec)\n",
      "INFO:tensorflow:nll_loss = 56.542747, kl_w = 0.99999905, kl_loss = 10.077927 (31.490 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.87266\n",
      "INFO:tensorflow:loss = 68.74959, step = 11449 (34.811 sec)\n",
      "INFO:tensorflow:nll_loss = 58.552406, kl_w = 0.9999993, kl_loss = 10.197193 (34.811 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.73143\n",
      "INFO:tensorflow:loss = 68.75374, step = 11549 (36.611 sec)\n",
      "INFO:tensorflow:nll_loss = 58.74369, kl_w = 0.9999994, kl_loss = 10.010056 (36.612 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.86202\n",
      "INFO:tensorflow:loss = 62.118366, step = 11649 (34.940 sec)\n",
      "INFO:tensorflow:nll_loss = 51.512993, kl_w = 0.9999994, kl_loss = 10.605379 (34.939 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 11730 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 65.56989.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-11730\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i think this movie and i have to be one of the worst movies <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie is one of those movies ever seen this movie if you have <end>\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-11730\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 11731 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:loss = 64.29591, step = 11731\n",
      "INFO:tensorflow:nll_loss = 53.901, kl_w = 0.9999995, kl_loss = 10.394917\n",
      "INFO:tensorflow:global_step/sec: 3.16656\n",
      "INFO:tensorflow:loss = 66.59424, step = 11831 (31.581 sec)\n",
      "INFO:tensorflow:nll_loss = 56.07374, kl_w = 0.99999964, kl_loss = 10.520504 (31.581 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.95834\n",
      "INFO:tensorflow:loss = 68.798035, step = 11931 (33.803 sec)\n",
      "INFO:tensorflow:nll_loss = 58.88025, kl_w = 0.99999964, kl_loss = 9.917789 (33.803 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.91395\n",
      "INFO:tensorflow:loss = 68.4216, step = 12031 (34.318 sec)\n",
      "INFO:tensorflow:nll_loss = 58.396267, kl_w = 0.99999976, kl_loss = 10.025337 (34.318 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.84369\n",
      "INFO:tensorflow:loss = 69.470436, step = 12131 (35.166 sec)\n",
      "INFO:tensorflow:nll_loss = 58.96074, kl_w = 0.99999976, kl_loss = 10.509701 (35.166 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.82584\n",
      "INFO:tensorflow:loss = 65.090355, step = 12231 (35.387 sec)\n",
      "INFO:tensorflow:nll_loss = 54.51951, kl_w = 0.99999976, kl_loss = 10.570847 (35.387 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.80189\n",
      "INFO:tensorflow:loss = 65.2483, step = 12331 (35.690 sec)\n",
      "INFO:tensorflow:nll_loss = 54.591827, kl_w = 0.9999999, kl_loss = 10.656471 (35.690 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.99933\n",
      "INFO:tensorflow:loss = 64.049286, step = 12431 (33.341 sec)\n",
      "INFO:tensorflow:nll_loss = 53.693295, kl_w = 0.9999999, kl_loss = 10.355993 (33.340 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 12512 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 65.521034.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-12512\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i love the first thing i think that is one of the worst movies <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie is a lot of time to watch this movie if you like <end>\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-12512\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 12513 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:loss = 63.758152, step = 12513\n",
      "INFO:tensorflow:nll_loss = 53.419044, kl_w = 0.9999999, kl_loss = 10.3391075\n",
      "INFO:tensorflow:global_step/sec: 2.89528\n",
      "INFO:tensorflow:loss = 68.1931, step = 12613 (34.541 sec)\n",
      "INFO:tensorflow:nll_loss = 57.959225, kl_w = 0.9999999, kl_loss = 10.233879 (34.540 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.99245\n",
      "INFO:tensorflow:loss = 66.89353, step = 12713 (33.417 sec)\n",
      "INFO:tensorflow:nll_loss = 56.489635, kl_w = 0.9999999, kl_loss = 10.403894 (33.417 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.23219\n",
      "INFO:tensorflow:loss = 66.621414, step = 12813 (30.939 sec)\n",
      "INFO:tensorflow:nll_loss = 56.140533, kl_w = 0.9999999, kl_loss = 10.480885 (30.939 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.87987\n",
      "INFO:tensorflow:loss = 67.71793, step = 12913 (34.724 sec)\n",
      "INFO:tensorflow:nll_loss = 57.617115, kl_w = 0.9999999, kl_loss = 10.100818 (34.724 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.93438\n",
      "INFO:tensorflow:loss = 63.34362, step = 13013 (34.078 sec)\n",
      "INFO:tensorflow:nll_loss = 52.43067, kl_w = 1.0, kl_loss = 10.912947 (34.079 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.11584\n",
      "INFO:tensorflow:loss = 63.85097, step = 13113 (32.094 sec)\n",
      "INFO:tensorflow:nll_loss = 53.086502, kl_w = 1.0, kl_loss = 10.764469 (32.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.99863\n",
      "INFO:tensorflow:loss = 63.079727, step = 13213 (33.350 sec)\n",
      "INFO:tensorflow:nll_loss = 52.422874, kl_w = 1.0, kl_loss = 10.656852 (33.350 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 13294 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 63.267147.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-13294\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i loved the first movie i thought it was the best of the best <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie is the first of mine i saw this movie to be a <end>\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-13294\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 13295 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:loss = 69.06355, step = 13295\n",
      "INFO:tensorflow:nll_loss = 58.443214, kl_w = 1.0, kl_loss = 10.620337\n",
      "INFO:tensorflow:global_step/sec: 2.93472\n",
      "INFO:tensorflow:loss = 66.974915, step = 13395 (34.076 sec)\n",
      "INFO:tensorflow:nll_loss = 56.52336, kl_w = 1.0, kl_loss = 10.45155 (34.076 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.96766\n",
      "INFO:tensorflow:loss = 66.2556, step = 13495 (33.697 sec)\n",
      "INFO:tensorflow:nll_loss = 55.642456, kl_w = 1.0, kl_loss = 10.613148 (33.696 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.09121\n",
      "INFO:tensorflow:loss = 66.46032, step = 13595 (32.350 sec)\n",
      "INFO:tensorflow:nll_loss = 56.06806, kl_w = 1.0, kl_loss = 10.39226 (32.351 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.83658\n",
      "INFO:tensorflow:loss = 63.74954, step = 13695 (35.254 sec)\n",
      "INFO:tensorflow:nll_loss = 52.930824, kl_w = 1.0, kl_loss = 10.818715 (35.253 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.93884\n",
      "INFO:tensorflow:loss = 63.212395, step = 13795 (34.026 sec)\n",
      "INFO:tensorflow:nll_loss = 52.245335, kl_w = 1.0, kl_loss = 10.967062 (34.026 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.00299\n",
      "INFO:tensorflow:loss = 63.043903, step = 13895 (33.301 sec)\n",
      "INFO:tensorflow:nll_loss = 51.969887, kl_w = 1.0, kl_loss = 11.074017 (33.301 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.16189\n",
      "INFO:tensorflow:loss = 62.43504, step = 13995 (31.626 sec)\n",
      "INFO:tensorflow:nll_loss = 51.36425, kl_w = 1.0, kl_loss = 11.070788 (31.626 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 14076 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 68.41821.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-14076\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i saw the first time i saw this was one of my favorite movies <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this is one of the worst movie ever made it is not to see <end>\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-14076\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 14077 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:loss = 65.15292, step = 14077\n",
      "INFO:tensorflow:nll_loss = 54.234844, kl_w = 1.0, kl_loss = 10.918081\n",
      "INFO:tensorflow:global_step/sec: 2.83678\n",
      "INFO:tensorflow:loss = 63.45938, step = 14177 (35.253 sec)\n",
      "INFO:tensorflow:nll_loss = 52.544594, kl_w = 1.0, kl_loss = 10.914787 (35.253 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.84078\n",
      "INFO:tensorflow:loss = 63.396935, step = 14277 (35.201 sec)\n",
      "INFO:tensorflow:nll_loss = 52.534294, kl_w = 1.0, kl_loss = 10.86264 (35.201 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.08253\n",
      "INFO:tensorflow:loss = 67.05235, step = 14377 (32.441 sec)\n",
      "INFO:tensorflow:nll_loss = 56.17409, kl_w = 1.0, kl_loss = 10.87826 (32.441 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.78332\n",
      "INFO:tensorflow:loss = 66.60458, step = 14477 (35.930 sec)\n",
      "INFO:tensorflow:nll_loss = 56.202515, kl_w = 1.0, kl_loss = 10.402065 (35.930 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.81815\n",
      "INFO:tensorflow:loss = 68.33344, step = 14577 (35.483 sec)\n",
      "INFO:tensorflow:nll_loss = 57.486576, kl_w = 1.0, kl_loss = 10.846867 (35.483 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.0174\n",
      "INFO:tensorflow:loss = 66.81008, step = 14677 (33.141 sec)\n",
      "INFO:tensorflow:nll_loss = 56.227592, kl_w = 1.0, kl_loss = 10.58249 (33.141 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.89927\n",
      "INFO:tensorflow:loss = 64.04544, step = 14777 (34.492 sec)\n",
      "INFO:tensorflow:nll_loss = 52.88591, kl_w = 1.0, kl_loss = 11.159529 (34.491 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 14858 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 62.301155.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-14858\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i loved this movie that i have seen it is the best of the <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this is one of the worst movie ever seen it is not to watch <end>\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-14858\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 14859 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:loss = 68.21974, step = 14859\n",
      "INFO:tensorflow:nll_loss = 57.32339, kl_w = 1.0, kl_loss = 10.896349\n",
      "INFO:tensorflow:global_step/sec: 2.78997\n",
      "INFO:tensorflow:loss = 67.44429, step = 14959 (35.844 sec)\n",
      "INFO:tensorflow:nll_loss = 56.792557, kl_w = 1.0, kl_loss = 10.6517315 (35.844 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.96691\n",
      "INFO:tensorflow:loss = 67.84619, step = 15059 (33.705 sec)\n",
      "INFO:tensorflow:nll_loss = 57.357605, kl_w = 1.0, kl_loss = 10.488583 (33.705 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.19506\n",
      "INFO:tensorflow:loss = 64.72456, step = 15159 (31.298 sec)\n",
      "INFO:tensorflow:nll_loss = 53.692635, kl_w = 1.0, kl_loss = 11.031932 (31.298 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.20203\n",
      "INFO:tensorflow:loss = 63.401142, step = 15259 (31.230 sec)\n",
      "INFO:tensorflow:nll_loss = 52.543816, kl_w = 1.0, kl_loss = 10.8573265 (31.230 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.04766\n",
      "INFO:tensorflow:loss = 63.203957, step = 15359 (32.812 sec)\n",
      "INFO:tensorflow:nll_loss = 52.243153, kl_w = 1.0, kl_loss = 10.960802 (32.812 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.76356\n",
      "INFO:tensorflow:loss = 63.192078, step = 15459 (36.185 sec)\n",
      "INFO:tensorflow:nll_loss = 52.26098, kl_w = 1.0, kl_loss = 10.931097 (36.185 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.91583\n",
      "INFO:tensorflow:loss = 67.77417, step = 15559 (34.296 sec)\n",
      "INFO:tensorflow:nll_loss = 57.296722, kl_w = 1.0, kl_loss = 10.47745 (34.296 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 15640 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 68.49401.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-15640\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i really liked this movie i think it was one of my favorite movies <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie is the worst movie ever made it is so i think it <end>\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-15640\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 15641 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:loss = 64.61891, step = 15641\n",
      "INFO:tensorflow:nll_loss = 53.928917, kl_w = 1.0, kl_loss = 10.689994\n",
      "INFO:tensorflow:global_step/sec: 3.2051\n",
      "INFO:tensorflow:loss = 68.03468, step = 15741 (31.201 sec)\n",
      "INFO:tensorflow:nll_loss = 57.43312, kl_w = 1.0, kl_loss = 10.601566 (31.201 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.22483\n",
      "INFO:tensorflow:loss = 67.0477, step = 15841 (31.009 sec)\n",
      "INFO:tensorflow:nll_loss = 56.18071, kl_w = 1.0, kl_loss = 10.866986 (31.009 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.1438\n",
      "INFO:tensorflow:loss = 66.97041, step = 15941 (31.809 sec)\n",
      "INFO:tensorflow:nll_loss = 56.483273, kl_w = 1.0, kl_loss = 10.487137 (31.809 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.18863\n",
      "INFO:tensorflow:loss = 62.2671, step = 16041 (31.361 sec)\n",
      "INFO:tensorflow:nll_loss = 50.907524, kl_w = 1.0, kl_loss = 11.359577 (31.361 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.92311\n",
      "INFO:tensorflow:loss = 60.910595, step = 16141 (34.210 sec)\n",
      "INFO:tensorflow:nll_loss = 49.661057, kl_w = 1.0, kl_loss = 11.249538 (34.210 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.75343\n",
      "INFO:tensorflow:loss = 62.75889, step = 16241 (36.318 sec)\n",
      "INFO:tensorflow:nll_loss = 51.84073, kl_w = 1.0, kl_loss = 10.9181595 (36.318 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.98461\n",
      "INFO:tensorflow:loss = 63.022663, step = 16341 (33.505 sec)\n",
      "INFO:tensorflow:nll_loss = 51.837975, kl_w = 1.0, kl_loss = 11.184688 (33.505 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 16422 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 68.071014.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-16422\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i loved this movie i can't believe it is one of my favorite movies <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie is a waste of time i've seen a lot to sit through <end>\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-16422\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 16423 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:loss = 66.3771, step = 16423\n",
      "INFO:tensorflow:nll_loss = 55.477703, kl_w = 1.0, kl_loss = 10.899393\n",
      "INFO:tensorflow:global_step/sec: 2.80556\n",
      "INFO:tensorflow:loss = 66.11226, step = 16523 (35.645 sec)\n",
      "INFO:tensorflow:nll_loss = 55.187656, kl_w = 1.0, kl_loss = 10.924601 (35.645 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.96816\n",
      "INFO:tensorflow:loss = 64.67673, step = 16623 (33.691 sec)\n",
      "INFO:tensorflow:nll_loss = 53.762253, kl_w = 1.0, kl_loss = 10.914475 (33.691 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.7908\n",
      "INFO:tensorflow:loss = 60.311344, step = 16723 (35.832 sec)\n",
      "INFO:tensorflow:nll_loss = 48.914627, kl_w = 1.0, kl_loss = 11.396717 (35.832 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.89528\n",
      "INFO:tensorflow:loss = 62.41993, step = 16823 (34.539 sec)\n",
      "INFO:tensorflow:nll_loss = 51.144005, kl_w = 1.0, kl_loss = 11.275923 (34.539 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.76824\n",
      "INFO:tensorflow:loss = 60.796547, step = 16923 (36.124 sec)\n",
      "INFO:tensorflow:nll_loss = 49.394997, kl_w = 1.0, kl_loss = 11.40155 (36.124 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.13474\n",
      "INFO:tensorflow:loss = 61.85247, step = 17023 (31.900 sec)\n",
      "INFO:tensorflow:nll_loss = 50.456627, kl_w = 1.0, kl_loss = 11.395842 (31.900 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.03233\n",
      "INFO:tensorflow:loss = 64.895676, step = 17123 (32.978 sec)\n",
      "INFO:tensorflow:nll_loss = 54.04074, kl_w = 1.0, kl_loss = 10.854931 (32.979 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 17204 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 68.11964.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-17204\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i saw this movie because i think it is one of the worst movies <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie is a waste of time i've seen it so much to watch <end>\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-17204\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 17205 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:loss = 65.33205, step = 17205\n",
      "INFO:tensorflow:nll_loss = 54.43625, kl_w = 1.0, kl_loss = 10.895799\n",
      "INFO:tensorflow:global_step/sec: 2.95024\n",
      "INFO:tensorflow:loss = 66.09587, step = 17305 (33.897 sec)\n",
      "INFO:tensorflow:nll_loss = 55.07727, kl_w = 1.0, kl_loss = 11.018599 (33.897 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.03085\n",
      "INFO:tensorflow:loss = 66.76487, step = 17405 (32.994 sec)\n",
      "INFO:tensorflow:nll_loss = 55.994717, kl_w = 1.0, kl_loss = 10.770151 (32.994 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.27307\n",
      "INFO:tensorflow:loss = 66.464745, step = 17505 (30.554 sec)\n",
      "INFO:tensorflow:nll_loss = 55.609383, kl_w = 1.0, kl_loss = 10.855363 (30.554 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.12478\n",
      "INFO:tensorflow:loss = 63.375477, step = 17605 (32.001 sec)\n",
      "INFO:tensorflow:nll_loss = 52.3491, kl_w = 1.0, kl_loss = 11.026379 (32.001 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.09324\n",
      "INFO:tensorflow:loss = 61.44909, step = 17705 (32.328 sec)\n",
      "INFO:tensorflow:nll_loss = 50.247803, kl_w = 1.0, kl_loss = 11.201286 (32.328 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.74947\n",
      "INFO:tensorflow:loss = 62.460922, step = 17805 (36.371 sec)\n",
      "INFO:tensorflow:nll_loss = 51.14974, kl_w = 1.0, kl_loss = 11.311186 (36.371 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.73603\n",
      "INFO:tensorflow:loss = 61.943558, step = 17905 (36.549 sec)\n",
      "INFO:tensorflow:nll_loss = 50.552727, kl_w = 1.0, kl_loss = 11.390833 (36.549 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 17986 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 64.46121.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-17986\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i loved this movie and i thought it was one of the worst films <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie is the worst movie ever made for this movie i have seen <end>\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-17986\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 17987 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:loss = 65.84075, step = 17987\n",
      "INFO:tensorflow:nll_loss = 54.716858, kl_w = 1.0, kl_loss = 11.123892\n",
      "INFO:tensorflow:global_step/sec: 2.9866\n",
      "INFO:tensorflow:loss = 60.51886, step = 18087 (33.484 sec)\n",
      "INFO:tensorflow:nll_loss = 49.123184, kl_w = 1.0, kl_loss = 11.395675 (33.484 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.62768\n",
      "INFO:tensorflow:loss = 62.402702, step = 18187 (38.056 sec)\n",
      "INFO:tensorflow:nll_loss = 51.18341, kl_w = 1.0, kl_loss = 11.219294 (38.056 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.37735\n",
      "INFO:tensorflow:loss = 60.27308, step = 18287 (42.066 sec)\n",
      "INFO:tensorflow:nll_loss = 48.859062, kl_w = 1.0, kl_loss = 11.414017 (42.066 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.77515\n",
      "INFO:tensorflow:loss = 60.465378, step = 18387 (36.032 sec)\n",
      "INFO:tensorflow:nll_loss = 49.186146, kl_w = 1.0, kl_loss = 11.279233 (36.032 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.00978\n",
      "INFO:tensorflow:loss = 66.6926, step = 18487 (33.225 sec)\n",
      "INFO:tensorflow:nll_loss = 55.926254, kl_w = 1.0, kl_loss = 10.76634 (33.225 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.03153\n",
      "INFO:tensorflow:loss = 66.207825, step = 18587 (32.987 sec)\n",
      "INFO:tensorflow:nll_loss = 55.297928, kl_w = 1.0, kl_loss = 10.909893 (32.987 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.0589\n",
      "INFO:tensorflow:loss = 68.33293, step = 18687 (32.691 sec)\n",
      "INFO:tensorflow:nll_loss = 57.4563, kl_w = 1.0, kl_loss = 10.87663 (32.691 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 18768 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 65.611916.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-18768\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i saw this movie because i think it was one of my favorite movies <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie is a great movie if you are a good time to watch <end>\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-18768\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 18769 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:loss = 63.811127, step = 18769\n",
      "INFO:tensorflow:nll_loss = 52.865837, kl_w = 1.0, kl_loss = 10.94529\n",
      "INFO:tensorflow:global_step/sec: 3.1153\n",
      "INFO:tensorflow:loss = 64.11142, step = 18869 (32.101 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:nll_loss = 52.75791, kl_w = 1.0, kl_loss = 11.353512 (32.101 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.06004\n",
      "INFO:tensorflow:loss = 62.172626, step = 18969 (32.679 sec)\n",
      "INFO:tensorflow:nll_loss = 50.66104, kl_w = 1.0, kl_loss = 11.511585 (32.679 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.38843\n",
      "INFO:tensorflow:loss = 62.3083, step = 19069 (41.870 sec)\n",
      "INFO:tensorflow:nll_loss = 51.093185, kl_w = 1.0, kl_loss = 11.215116 (41.871 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.35495\n",
      "INFO:tensorflow:loss = 61.315468, step = 19169 (42.463 sec)\n",
      "INFO:tensorflow:nll_loss = 49.836723, kl_w = 1.0, kl_loss = 11.4787445 (42.462 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.43249\n",
      "INFO:tensorflow:loss = 65.64402, step = 19269 (41.111 sec)\n",
      "INFO:tensorflow:nll_loss = 54.46495, kl_w = 1.0, kl_loss = 11.1790695 (41.111 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.82874\n",
      "INFO:tensorflow:loss = 64.85683, step = 19369 (35.350 sec)\n",
      "INFO:tensorflow:nll_loss = 53.872993, kl_w = 1.0, kl_loss = 10.983837 (35.349 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.91957\n",
      "INFO:tensorflow:loss = 66.08677, step = 19469 (34.252 sec)\n",
      "INFO:tensorflow:nll_loss = 55.086143, kl_w = 1.0, kl_loss = 11.0006275 (34.252 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 19550 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 62.278145.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-19550\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i loved this movie and i think it is one of the worst movies <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie is a waste of time to watch it to have been on <end>\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-19550\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 19551 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:loss = 61.415962, step = 19551\n",
      "INFO:tensorflow:nll_loss = 49.64349, kl_w = 1.0, kl_loss = 11.772472\n",
      "INFO:tensorflow:global_step/sec: 2.74974\n",
      "INFO:tensorflow:loss = 65.92438, step = 19651 (36.368 sec)\n",
      "INFO:tensorflow:nll_loss = 54.857567, kl_w = 1.0, kl_loss = 11.066808 (36.368 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.12244\n",
      "INFO:tensorflow:loss = 64.44838, step = 19751 (32.026 sec)\n",
      "INFO:tensorflow:nll_loss = 53.43702, kl_w = 1.0, kl_loss = 11.01136 (32.026 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.72701\n",
      "INFO:tensorflow:loss = 66.40949, step = 19851 (36.671 sec)\n",
      "INFO:tensorflow:nll_loss = 55.284992, kl_w = 1.0, kl_loss = 11.124502 (36.672 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.72897\n",
      "INFO:tensorflow:loss = 67.809204, step = 19951 (36.644 sec)\n",
      "INFO:tensorflow:nll_loss = 56.895123, kl_w = 1.0, kl_loss = 10.914084 (36.643 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.71808\n",
      "INFO:tensorflow:loss = 60.98729, step = 20051 (36.790 sec)\n",
      "INFO:tensorflow:nll_loss = 49.56814, kl_w = 1.0, kl_loss = 11.419149 (36.791 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.97032\n",
      "INFO:tensorflow:loss = 63.596962, step = 20151 (33.666 sec)\n",
      "INFO:tensorflow:nll_loss = 52.16694, kl_w = 1.0, kl_loss = 11.430022 (33.666 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.76451\n",
      "INFO:tensorflow:loss = 62.182953, step = 20251 (36.173 sec)\n",
      "INFO:tensorflow:nll_loss = 50.810802, kl_w = 1.0, kl_loss = 11.372152 (36.173 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 20332 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 60.622406.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-20332\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i loved this movie and i was so much about the the first of <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie is a very funny movie it is a good if you have <end>\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-20332\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 20333 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:loss = 60.61914, step = 20333\n",
      "INFO:tensorflow:nll_loss = 48.996635, kl_w = 1.0, kl_loss = 11.622503\n",
      "INFO:tensorflow:global_step/sec: 2.84301\n",
      "INFO:tensorflow:loss = 61.86564, step = 20433 (35.175 sec)\n",
      "INFO:tensorflow:nll_loss = 50.555332, kl_w = 1.0, kl_loss = 11.310307 (35.175 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.7782\n",
      "INFO:tensorflow:loss = 59.073086, step = 20533 (35.995 sec)\n",
      "INFO:tensorflow:nll_loss = 47.391712, kl_w = 1.0, kl_loss = 11.681374 (35.995 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.8147\n",
      "INFO:tensorflow:loss = 65.862015, step = 20633 (35.528 sec)\n",
      "INFO:tensorflow:nll_loss = 54.653923, kl_w = 1.0, kl_loss = 11.208089 (35.528 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.5826\n",
      "INFO:tensorflow:loss = 65.044334, step = 20733 (38.721 sec)\n",
      "INFO:tensorflow:nll_loss = 53.94995, kl_w = 1.0, kl_loss = 11.094383 (38.721 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.70782\n",
      "INFO:tensorflow:loss = 65.50371, step = 20833 (36.930 sec)\n",
      "INFO:tensorflow:nll_loss = 54.3295, kl_w = 1.0, kl_loss = 11.1742115 (36.930 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.61908\n",
      "INFO:tensorflow:loss = 65.4541, step = 20933 (38.181 sec)\n",
      "INFO:tensorflow:nll_loss = 54.28566, kl_w = 1.0, kl_loss = 11.168442 (38.181 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.71045\n",
      "INFO:tensorflow:loss = 63.81823, step = 21033 (36.894 sec)\n",
      "INFO:tensorflow:nll_loss = 52.469666, kl_w = 1.0, kl_loss = 11.348566 (36.894 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 21114 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 61.945538.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-21114\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i loved this movie because i have seen it in the most interesting films <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie is a very good movie that it is one of all the <end>\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-21114\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 21115 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:loss = 67.92, step = 21115\n",
      "INFO:tensorflow:nll_loss = 56.370705, kl_w = 1.0, kl_loss = 11.549295\n",
      "INFO:tensorflow:global_step/sec: 2.63941\n",
      "INFO:tensorflow:loss = 65.41459, step = 21215 (37.889 sec)\n",
      "INFO:tensorflow:nll_loss = 54.30359, kl_w = 1.0, kl_loss = 11.111003 (37.889 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.7641\n",
      "INFO:tensorflow:loss = 65.3991, step = 21315 (36.177 sec)\n",
      "INFO:tensorflow:nll_loss = 54.27224, kl_w = 1.0, kl_loss = 11.126861 (36.177 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 2.73048\n",
      "INFO:tensorflow:loss = 64.149254, step = 21415 (36.624 sec)\n",
      "INFO:tensorflow:nll_loss = 52.80629, kl_w = 1.0, kl_loss = 11.342966 (36.626 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.81947\n",
      "INFO:tensorflow:loss = 62.412476, step = 21515 (35.467 sec)\n",
      "INFO:tensorflow:nll_loss = 50.978462, kl_w = 1.0, kl_loss = 11.434011 (35.466 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.82561\n",
      "INFO:tensorflow:loss = 60.556, step = 21615 (35.391 sec)\n",
      "INFO:tensorflow:nll_loss = 48.821335, kl_w = 1.0, kl_loss = 11.734667 (35.391 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.67037\n",
      "INFO:tensorflow:loss = 62.503704, step = 21715 (37.448 sec)\n",
      "INFO:tensorflow:nll_loss = 50.98715, kl_w = 1.0, kl_loss = 11.516556 (37.449 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.83202\n",
      "INFO:tensorflow:loss = 60.516922, step = 21815 (35.310 sec)\n",
      "INFO:tensorflow:nll_loss = 49.139114, kl_w = 1.0, kl_loss = 11.37781 (35.310 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 21896 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 63.703014.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-21896\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i loved this movie i thought it was a lot of the best movies <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie is a waste of time it is so bad i have seen <end>\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-21896\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 21897 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:loss = 59.94249, step = 21897\n",
      "INFO:tensorflow:nll_loss = 48.32754, kl_w = 1.0, kl_loss = 11.614947\n",
      "INFO:tensorflow:global_step/sec: 2.80397\n",
      "INFO:tensorflow:loss = 65.09961, step = 21997 (35.665 sec)\n",
      "INFO:tensorflow:nll_loss = 53.788322, kl_w = 1.0, kl_loss = 11.311288 (35.665 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.95744\n",
      "INFO:tensorflow:loss = 65.3676, step = 22097 (33.813 sec)\n",
      "INFO:tensorflow:nll_loss = 54.228516, kl_w = 1.0, kl_loss = 11.139083 (33.813 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.88432\n",
      "INFO:tensorflow:loss = 63.103783, step = 22197 (34.670 sec)\n",
      "INFO:tensorflow:nll_loss = 51.600883, kl_w = 1.0, kl_loss = 11.5029 (34.670 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.56926\n",
      "INFO:tensorflow:loss = 64.707016, step = 22297 (38.923 sec)\n",
      "INFO:tensorflow:nll_loss = 53.483177, kl_w = 1.0, kl_loss = 11.223838 (38.924 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.69464\n",
      "INFO:tensorflow:loss = 58.78703, step = 22397 (37.110 sec)\n",
      "INFO:tensorflow:nll_loss = 47.107346, kl_w = 1.0, kl_loss = 11.679684 (37.109 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.79494\n",
      "INFO:tensorflow:loss = 61.78607, step = 22497 (35.778 sec)\n",
      "INFO:tensorflow:nll_loss = 50.025642, kl_w = 1.0, kl_loss = 11.760431 (35.778 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.93854\n",
      "INFO:tensorflow:loss = 60.75968, step = 22597 (34.030 sec)\n",
      "INFO:tensorflow:nll_loss = 49.275078, kl_w = 1.0, kl_loss = 11.484604 (34.030 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 22678 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 62.6267.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-22678\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i loved this movie when it was so much in the most interesting movie <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie is a lot of those that had no reason to watch the <end>\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-22678\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 22679 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:loss = 62.378933, step = 22679\n",
      "INFO:tensorflow:nll_loss = 50.642544, kl_w = 1.0, kl_loss = 11.736389\n",
      "INFO:tensorflow:global_step/sec: 2.96006\n",
      "INFO:tensorflow:loss = 59.697456, step = 22779 (33.786 sec)\n",
      "INFO:tensorflow:nll_loss = 47.751556, kl_w = 1.0, kl_loss = 11.945898 (33.787 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.89476\n",
      "INFO:tensorflow:loss = 60.09896, step = 22879 (34.543 sec)\n",
      "INFO:tensorflow:nll_loss = 48.31878, kl_w = 1.0, kl_loss = 11.780181 (34.543 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.94794\n",
      "INFO:tensorflow:loss = 61.053993, step = 22979 (33.922 sec)\n",
      "INFO:tensorflow:nll_loss = 49.192406, kl_w = 1.0, kl_loss = 11.861587 (33.922 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.18681\n",
      "INFO:tensorflow:loss = 66.286316, step = 23079 (31.379 sec)\n",
      "INFO:tensorflow:nll_loss = 54.748222, kl_w = 1.0, kl_loss = 11.538097 (31.379 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.98673\n",
      "INFO:tensorflow:loss = 64.47404, step = 23179 (33.482 sec)\n",
      "INFO:tensorflow:nll_loss = 53.06788, kl_w = 1.0, kl_loss = 11.406161 (33.482 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.77063\n",
      "INFO:tensorflow:loss = 65.62447, step = 23279 (36.093 sec)\n",
      "INFO:tensorflow:nll_loss = 54.312084, kl_w = 1.0, kl_loss = 11.312391 (36.093 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.07142\n",
      "INFO:tensorflow:loss = 66.67821, step = 23379 (32.559 sec)\n",
      "INFO:tensorflow:nll_loss = 55.376957, kl_w = 1.0, kl_loss = 11.301249 (32.558 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 23460 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 61.340855.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp88gb9b9a/model.ckpt-23460\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i loved this movie and i have seen it in the the main characters <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie was a waste of time it was so bad i have seen <end>\n",
      "\n"
     ]
    }
   ],
   "source": [
    "main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
